import scrapy
from testdemo.items import TopicItem


class HpBxjSpider(scrapy.Spider):
    name = 'hp_bxj'
    allowed_domains = ['bbs.hupu.com']
    start_urls = ['https://bbs.hupu.com/bxj']

    def parse(self, response):
        topics = response.xpath('//ul[@class="for-list"]')
        for item in topics.xpath('li'):
            yield scrapy.Request(
                url='https://bbs.hupu.com' + item.xpath('div/a//@href').get(),
                callback=self.parse_topic
            )

    def parse_topic(self, response):
        yield TopicItem(
            name=response.xpath('/html/head/title/text()').get(),
            url=response.url
        )
